package cn.com.lyb.flink.sink;

import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.serialization.SimpleStringEncoder;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.configuration.MemorySize;
import org.apache.flink.connector.datagen.source.DataGeneratorSource;
import org.apache.flink.connector.datagen.source.GeneratorFunction;
import org.apache.flink.connector.file.sink.FileSink;
import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig;
import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.DateTimeBucketAssigner;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy;

import java.time.Duration;
import java.time.ZoneId;

public class SinkFileDemo {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // 每个目录中都有并行度个数的写入
        env.setParallelism(8);
        //  必须开启Checkpointing， 不然文件一直都是inprogress
        env.enableCheckpointing(2000, CheckpointingMode.EXACTLY_ONCE);
        // DataGeneratorSource接口 需要实现，重写map方法，输入类型固定是Long
        DataGeneratorSource<String> stringDataGeneratorSource = new DataGeneratorSource<>(
                new GeneratorFunction<Long, String>() {
                    @Override
                    public String map(Long value) throws Exception {
                        return "Number: " + value;
                    }
                },
                1000000, // 最大值（-1），自动生成的数字序列，从0自增，步长为1
                RateLimiterStrategy.perSecond(1000), // 限速策略每秒生成多少个，限流
                Types.STRING // 返回的类型
        );
        DataStreamSource<String> dateGenSource = env.fromSource(stringDataGeneratorSource, WatermarkStrategy.noWatermarks(), "dateGen");

        // 输出文件系统
        FileSink<String> fileSink = FileSink.
                <String>forRowFormat(new Path("D:\\logs\\flink"), new SimpleStringEncoder<>("UTF-8"))
                .withOutputFileConfig( // 输出文件的配置，文件名的前缀和后缀
                        OutputFileConfig.builder()
                                .withPartPrefix("qianzhui")
                                .withPartSuffix("houzhui.log")
                                .build())
                // 按照目录分桶  yyyy-MM-dd  HH ——> 就是目录，这个目录也会自动变
                .withBucketAssigner(new DateTimeBucketAssigner<>("yyyy-MM-dd  HH", ZoneId.systemDefault()))// 文件分桶
                // 文件滚动策略  10 或 1024*1024 = 1m
                .withRollingPolicy(DefaultRollingPolicy.builder()
                        .withRolloverInterval(Duration.ofSeconds(30))
                        .withMaxPartSize(new MemorySize(1024 * 1024)).
                        build())
                .build();

        dateGenSource.sinkTo(fileSink);
        env.execute();
























        env.execute();
    }
}
